package com.vivioning.region;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.brucezee.jspider.*;
import com.brucezee.jspider.pipeline.Pipeline;
import com.brucezee.jspider.processor.PageProcessor;
import com.brucezee.jspider.scheduler.QueueScheduler;
import com.github.stuxuhai.jpinyin.PinyinException;
import com.github.stuxuhai.jpinyin.PinyinFormat;
import com.github.stuxuhai.jpinyin.PinyinHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * AMapSpider.java
 *
 * @author XinJie.Guo
 * @date 2019/4/8
 */
public class AMapSpider {
    private static final Logger logger = LoggerFactory.getLogger(AMapSpider.class);

    /**
     * 数据
     */
    private static List<Region> regionList = new ArrayList<>();

    /**
     * 起始地址  ${你的高德key}
     */
    private static String startUrl = "https://restapi.amap.com/v3/config/district?key=${你的高德地图的Key}&output=json&subdistrict=4&extensions=base&keywords=";

    /**
     * 全局spider
     */
    private static Spider spider;

    public static void main(String[] args) {
        init();
    }

    protected static void init() {
        SpiderConfig spiderConfig = SpiderConfig.create("amap_gaode", 1)
                .setExitWhenComplete(true)
                .setDestroyWhenExit(true)
                .setEmptySleepMillis(30000)
                .setCloseDelayMillis(60000);

        SiteConfig siteConfig = SiteConfig.create()
                .setConnectionRequestTimeout(15000)
                .setConnectTimeout(30000)
                .setSocketTimeout(45000)
                .setSoKeepAlive(true)
                .setTcpNoDelay(true)
                .setBufferSize(8192)
                .setMaxIdleTimeMillis(1000 * 60 * 60)
                .setMaxConnTotal(200)
                .setMaxConnPerRoute(100)
                .setConnTimeToLiveMillis(-1)
                .setRedirectsEnabled(true)
                .setRelativeRedirectsAllowed(true)
                .setCircularRedirectsAllowed(false)
                .setMaxRedirects(3)
                .putCharset("amap", "utf8");

        //create, config and start
        spider = Spider.create(spiderConfig, siteConfig, new NotePageProcessor())
                .setPipeline(new NotePipeline())
                .setScheduler(new NoteScheduler())
//                .setHttpProxyPool(new HttpProxyPoolImpl())
                .addStartRequests(startUrl);

        spider.run();
    }

    protected static class NotePageProcessor implements PageProcessor {

        @Override
        public Result process(Request request, Page page) {
            Result result = new Result();
            GDResponse gdResponse = JSONObject.parseObject(page.getResult().toString(), GDResponse.class);
            if (gdResponse.getStatus().equals("1") && gdResponse.getInfo().equals("OK") && gdResponse.getInfocode().equals("10000")) {
                //获取中华人民共和国
                District country = JSONArray.parseArray(gdResponse.getDistricts(), District.class).get(0);
                Region countryRegion = new Region();
                countryRegion.setCity_code(country.getAdcode());
                countryRegion.setName(country.getName());
                countryRegion.setMerger_name(country.getName());
                countryRegion.setShort_name("中国");
                countryRegion.setMerger_short_name("中国");
                countryRegion.setLevel(country.getLevel());
                countryRegion.setLevel_type(0);
                //全拼
                countryRegion.setName_pinyin(getCode(country.getName()));
                //首字母
                countryRegion.setName_jianpin(getFirstCode(country.getName()));
                countryRegion.setName_first_char(getFirstCode(country.getName().substring(0, 1)));
                countryRegion.setCenter(country.getCenter());
                List<String> lal = Arrays.asList(country.getCenter().split(","));
                countryRegion.setLongitude(lal.get(0));
                countryRegion.setLatitude(lal.get(1));
                regionList.add(countryRegion);
                //获取省级城市列表
                List<District> provinceList = JSONObject.parseArray(country.getDistricts(), District.class);
                for (District province : provinceList) {
                    //获取省级城市数据
                    Region provinceRegion = new Region();
                    provinceRegion.setCity_code(province.getAdcode());
                    provinceRegion.setName(province.getName());
                    provinceRegion.setMerger_name(province.getName());
                    provinceRegion.setShort_name(province.getName().substring(0, province.getName().length() - 1));
                    provinceRegion.setMerger_short_name(province.getName().substring(0, province.getName().length() - 1));
                    provinceRegion.setLevel(province.getLevel());
                    provinceRegion.setLevel_type(1);
                    //全拼
                    provinceRegion.setName_pinyin(getCode(province.getName()));
                    //首字母
                    provinceRegion.setName_jianpin(getFirstCode(province.getName()));
                    provinceRegion.setName_first_char(getFirstCode(province.getName().substring(0, 1)));
                    provinceRegion.setCenter(province.getCenter());
                    List<String> provinceLal = Arrays.asList(province.getCenter().split(","));
                    provinceRegion.setLongitude(provinceLal.get(0));
                    provinceRegion.setLatitude(provinceLal.get(1));
                    regionList.add(provinceRegion);
                    //获取市级城市数据列表
                    List<District> cityList = JSONObject.parseArray(province.getDistricts(), District.class);

                    for (District city : cityList) {
                        //获取市级城市数据
                        Region cityRegion = new Region();
                        cityRegion.setCity_code(city.getAdcode());
                        cityRegion.setParent_code(province.getAdcode());
                        cityRegion.setName(city.getName());
                        cityRegion.setMerger_name(province.getName() + "," + city.getName());
                        cityRegion.setShort_name(city.getName().substring(0, city.getName().length() - 1));
                        cityRegion.setMerger_short_name(
                                province.getName().substring(0, province.getName().length() - 1) + "," +
                                        city.getName().substring(0, city.getName().length() - 1));
                        cityRegion.setLevel(city.getLevel());
                        cityRegion.setLevel_type(2);
                        cityRegion.setTelephone_code(city.getCitycode());
                        //全拼
                        cityRegion.setName_pinyin(getCode(city.getName()));
                        //首字母
                        cityRegion.setName_jianpin(getFirstCode(city.getName()));
                        cityRegion.setName_first_char(getFirstCode(city.getName().substring(0, 1)));
                        cityRegion.setCenter(city.getCenter());
                        List<String> cityLal = Arrays.asList(city.getCenter().split(","));
                        cityRegion.setLongitude(cityLal.get(0));
                        cityRegion.setLatitude(cityLal.get(1));
                        regionList.add(cityRegion);
                        //获取区级数据列表
                        List<District> districtList = JSONObject.parseArray(city.getDistricts(), District.class);
                        //获取区级城市数据
                        for (District district : districtList) {
                            //获取区级城市数据
                            Region districtRegion = new Region();
                            districtRegion.setCity_code(district.getAdcode());
                            districtRegion.setParent_code(city.getAdcode());
                            districtRegion.setName(district.getName());
                            districtRegion.setMerger_name(province.getName() + "," + city.getName() + "," + district.getName());
                            districtRegion.setShort_name(district.getName().substring(0, district.getName().length() - 1));
                            districtRegion.setMerger_short_name(
                                    province.getName().substring(0, province.getName().length() - 1) + "," +
                                            city.getName().substring(0, city.getName().length() - 1) + "," +
                                            district.getName().substring(0, district.getName().length() - 1));
                            districtRegion.setLevel(district.getLevel());
                            districtRegion.setLevel_type(3);
                            districtRegion.setTelephone_code(district.getCitycode());
                            //全拼
                            districtRegion.setName_pinyin(getCode(district.getName()));
                            //首字母
                            districtRegion.setName_jianpin(getFirstCode(district.getName()));
                            districtRegion.setName_first_char(getFirstCode(district.getName().substring(0, 1)));
                            districtRegion.setCenter(district.getCenter());
                            List<String> districtLal = Arrays.asList(district.getCenter().split(","));
                            if (districtLal.size() > 0) {
                                districtRegion.setLongitude(districtLal.get(0));
                                districtRegion.setLatitude(districtLal.get(1));
                            } else {
                                logger.error("查找区级城市的经纬度: {} 异常", district.getName());
                            }

                            regionList.add(districtRegion);
                            //获取乡镇数据列表
                            List<District> towmList = JSONObject.parseArray(district.getDistricts(), District.class);
                            for (District towm : towmList) {
                                if (!towm.getDistricts().equals("[]")) {
                                    logger.error("异常乡镇数据：{} ", towm.getName());
                                    List<District> specialTown = JSONObject.parseArray(towm.getDistricts(), District.class);
                                    if (specialTown.size() == 1) {
                                        towm = specialTown.get(0);
                                    }
                                }
                                //获取乡镇数据
                                Region townRegion = new Region();
                                townRegion.setCity_code(towm.getAdcode());
                                townRegion.setParent_code(district.getAdcode());
                                townRegion.setName(towm.getName());
                                townRegion.setMerger_name(province.getName() + "," + city.getName() + "," + district.getName() + "," + towm.getName());
                                townRegion.setShort_name(towm.getName().substring(0, towm.getName().length() - 1));
                                townRegion.setMerger_short_name(
                                        province.getName().substring(0, province.getName().length() - 1) + "," +
                                                city.getName().substring(0, city.getName().length() - 1) + "," +
                                                district.getName().substring(0, district.getName().length() - 1) + "," +
                                                towm.getName().substring(0, towm.getName().length() - 1));
                                townRegion.setLevel(towm.getLevel());
                                townRegion.setLevel_type(4);
                                townRegion.setTelephone_code(towm.getCitycode());
                                //全拼
                                townRegion.setName_pinyin(getCode(towm.getName()));
                                //首字母
                                townRegion.setName_jianpin(getFirstCode(towm.getName()));
                                townRegion.setName_first_char(getFirstCode(towm.getName().substring(0, 1)));
                                townRegion.setCenter(towm.getCenter());
                                List<String> towmLal = Arrays.asList(towm.getCenter().split(","));
                                if (towmLal.size() > 0) {
                                    townRegion.setLongitude(towmLal.get(0));
                                    townRegion.setLatitude(towmLal.get(1));
                                } else {
                                    logger.error("查找乡镇的经纬度: {} 异常", towm.getName());
                                }
                                regionList.add(townRegion);
                            }
                        }
                    }

                }

            }
            result.put("数据", regionList);
            return result;
        }
    }

    private static class NotePipeline implements Pipeline {

        @Override
        public void persist(Request request, Result result) {

            logger.info("----------------------华丽的分割线---------------------");
            logger.info("总计：{}", regionList.size());
            logger.info("----------------------华丽的分割线---------------------");
            logger.info("文本输出开始: ");
            try {
                File destFile = new File("D:\\\\region.sql");
                // 目标文件存在则删除
                if (!destFile.exists()) {
                    destFile.delete();
                }
                destFile.createNewFile();
                OutputStreamWriter write = new OutputStreamWriter(new FileOutputStream(destFile), "utf-8");
                BufferedWriter fw = new BufferedWriter(write);

                fw.write("-- 省市区乡镇4及联动");
                fw.write("\r\n");
                fw.write("-- 更新时间：2019-1-15");
                fw.write("\r\n");
                fw.write("\r\n");
                fw.write("-- ----------------------------");
                fw.write("\r\n");
                fw.write("-- Table structure for tb_region");
                fw.write("\r\n");
                fw.write("-- ----------------------------");
                fw.write("\r\n");
                fw.write("DROP TABLE IF EXISTS `tb_region`;");
                fw.write("\r\n");
                fw.write("CREATE TABLE `tb_region` (");
                fw.write("\r\n");
                fw.write("  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '唯一性编号',");
                fw.write("\r\n");
                fw.write("  `city_code` varchar(32) NOT NULL COMMENT '行政区划id',");
                fw.write("\r\n");
                fw.write("  `parent_code` varchar(32) DEFAULT NULL COMMENT '父级id',");
                fw.write("\r\n");
                fw.write("  `name` varchar(256) NOT NULL COMMENT '行政区划全称',");
                fw.write("\r\n");
                fw.write("  `merger_name` varchar(256) NOT NULL COMMENT '省市区全称聚合',");
                fw.write("\r\n");
                fw.write("  `short_name` varchar(48) NOT NULL COMMENT '行政区划简称',");
                fw.write("\r\n");
                fw.write("  `merger_short_name` varchar(200) NOT NULL COMMENT '行政区划简称聚合',");
                fw.write("\r\n");
                fw.write("  `level` varchar(16) NOT NULL COMMENT '行政区划级别country:国家,province:省份,city:市,district:区县,street:街道',");
                fw.write("\r\n");
                fw.write("  `level_type` tinyint(3) NOT NULL COMMENT '级别 0.国家，1.省(直辖市) 2.市 3.区(县),4.街道',");
                fw.write("\r\n");
                fw.write("  `telephone_code` varchar(32) DEFAULT NULL COMMENT '电话区划号码',");
                fw.write("\r\n");
                fw.write("  `zip_code` varchar(32) DEFAULT NULL COMMENT '邮编',");
                fw.write("\r\n");
                fw.write("  `name_pinyin` varchar(256) NOT NULL COMMENT '拼音',");
                fw.write("\r\n");
                fw.write("  `name_jianpin` varchar(64) NOT NULL COMMENT '简拼',");
                fw.write("\r\n");
                fw.write("  `center` varchar(32) NOT NULL COMMENT '城市中心点',");
                fw.write("\r\n");
                fw.write("  `name_first_char` varchar(64) DEFAULT NULL COMMENT '首字母',");
                fw.write("\r\n");
                fw.write("  `longitude` varchar(32) DEFAULT NULL COMMENT '经度',");
                fw.write("\r\n");
                fw.write("  `latitude` varchar(32) DEFAULT NULL COMMENT '纬度',");
                fw.write("\r\n");
                fw.write("  `status` tinyint(3) NOT NULL DEFAULT '1' COMMENT '状态 1可修改 2不可修改 3已删除',");
                fw.write("\r\n");
                fw.write("  `version` varchar(16) NOT NULL COMMENT '历史版本',");
                fw.write("\r\n");
                fw.write("  `create_time` datetime DEFAULT NULL COMMENT '创建时间',");
                fw.write("\r\n");
                fw.write("  `update_time` datetime DEFAULT NULL COMMENT '更新时间',");
                fw.write("\r\n");
                fw.write("  PRIMARY KEY (`id`) USING BTREE");
                fw.write("\r\n");
                fw.write(") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC COMMENT='中国行政地址信息表';");
                fw.write("\r\n");
                fw.write("-- ----------------------------");
                fw.write("\r\n");
                fw.write("-- Records of tb_region");
                fw.write("\r\n");
                fw.write("-- ----------------------------");
                fw.write("\r\n");
                fw.write("\r\n");

                for (Region region : regionList) {
                    StringBuilder regionText = new StringBuilder("INSERT INTO `tb_bb_region`(`city_code`, `parent_code`, `name`, `merger_name`, `short_name`, `merger_short_name`, `level`, `level_type`, `telephone_code`, `zip_code`, `name_pinyin`, `name_jianpin`, `center`, `name_first_char`, `longitude`, `latitude`, `status`, `version`, `create_time`, `update_time`) VALUES(");
                    regionText = regionText.append("\"").append(region.getCity_code() != null ? region.getCity_code() : "null").append("\"").append(",")
                            .append("\"").append(region.getParent_code() != null ? region.getParent_code() : "null").append("\"").append(",")
                            .append("\"").append(region.getName() != null ? region.getName() : "null").append("\"").append(",")
                            .append("\"").append(region.getMerger_name() != null ? region.getMerger_name() : "null").append("\"").append(",")
                            .append("\"").append(region.getShort_name() != null ? region.getShort_name() : "null").append("\"").append(",")
                            .append("\"").append(region.getMerger_short_name() != null ? region.getMerger_short_name() : "null").append("\"").append(",")
                            .append("\"").append(region.getLevel() != null ? region.getLevel() : "null").append("\"").append(",")
                            .append(region.getLevel_type()).append(",")
                            .append("\"").append(region.getTelephone_code() != null ? region.getTelephone_code() : "null").append("\"").append(",")
                            .append("\"").append(region.getZip_code() != null ? region.getZip_code() : "null").append("\"").append(",")
                            .append("\"").append(region.getName_pinyin() != null ? region.getName_pinyin() : "null").append("\"").append(",")
                            .append("\"").append(region.getName_jianpin() != null ? region.getName_jianpin() : "null").append("\"").append(",")
                            .append("\"").append(region.getName_first_char() != null ? region.getName_first_char() : "null").append("\"").append(",")
                            .append("\"").append(region.getCenter() != null ? region.getCenter() : "null").append("\"").append(",")
                            .append("\"").append(region.getLongitude() != null ? region.getLongitude() : "null").append("\"").append(",")
                            .append("\"").append(region.getLatitude() != null ? region.getLatitude() : "null").append("\"").append(",")
                            .append(1).append(",")
                            .append("\"20190115\"").append(",")
                            .append("NOW()").append(",")
                            .append("NOW()")
                            .append(");");
                    fw.write(regionText.toString().replaceAll("\"null\"", "null"));
                    fw.write("\r\n");
                    logger.info("地址：{} 写入结束", region.getName());
                    logger.info("----------------------分割线---------------------");
                }
                fw.close();
            } catch (IOException e1) {
                e1.printStackTrace();
            }

        }
    }

    private static class NoteScheduler extends QueueScheduler {
        @Override
        public int getLeftCount(Task task) {

            return super.getLeftCount(task);
        }

    }


    /**
     * 获取全拼
     *
     * @param name
     * @return
     */
    private static String getCode(String name) {
        String code = null;
        try {
            code = PinyinHelper.convertToPinyinString(name, "", PinyinFormat.WITHOUT_TONE);
        } catch (PinyinException e) {
            e.printStackTrace();
        }
        return code;
    }

    /**
     * 获取首字母
     *
     * @param name
     * @return
     */
    private static String getFirstCode(String name) {
        String code = null;
        try {
            code = PinyinHelper.getShortPinyin(name);
        } catch (PinyinException e) {
            e.printStackTrace();
        }
        return code;
    }
}
